This notebook compares the results from assigning consensus cell
types with SCimilarity to the previous consensus cell types
without SCimilarity. All results from the
cell-type-consensus module in OpenScPCA-nf
must be saved to results prior to rendering this
notebook.
Note that any multiplexed libraries have been excluded from this
notebook as SCimilarity is not run on those libraries when
using the OpenScPCA-nf module.
suppressPackageStartupMessages({
# load required packages
library(ggplot2)
})
# Set default ggplot theme
theme_set(
theme_classic()
)
# Define color ramp for shared use in the heatmaps
heatmap_col_fun <- circlize::colorRamp2(c(0, 1), colors = c("white", "darkslateblue"))
# Set heatmap padding option
ComplexHeatmap::ht_opt(TITLE_PADDING = grid::unit(0.6, "in"))
# settings
options(
dplyr.summarise.inform = FALSE,
readr.show_col_types = FALSE
)
Data setup
# The base path for the OpenScPCA repository, found by its (hidden) .git directory
repository_base <- rprojroot::find_root(rprojroot::is_git_root)
module_base <- file.path(repository_base, "analyses", "cell-type-consensus")
# results directory with cell-type-consensus
results_dir <- file.path(module_base, "results", "cell-type-consensus")
# diagnoses table used for labeling plots
diagnoses_file <- file.path(module_base, "sample-info", "project-diagnoses.tsv")
# use the jaccard functions from cell-type-neuroblastoma-04 module
jaccard_functions <- file.path(repository_base, "analyses", "cell-type-neuroblastoma-04", "scripts", "utils", "jaccard-utils.R")
source(jaccard_functions)
# list all results files
results_files <- list.files(results_dir, pattern = "_consensus-cell-types\\.tsv.\\gz$", recursive = TRUE, full.names = TRUE)
# define cell line projects to remove
cell_line_projects <- c("SCPCP000020", "SCPCP000024")
# read in diagnoses
diagnoses_df <- readr::read_tsv(diagnoses_file)
# read in results and prep data frame for plotting
all_results_df <- results_files |>
purrr::map(readr::read_tsv, col_types = "c")|>
dplyr::bind_rows() |>
# remove cell line projects
dplyr::filter(!project_id %in% cell_line_projects,
# remove multiplexed libraries since we didn't run scimilarity on them
!stringr::str_detect(sample_id, ";")) |>
# add in diagnoses
dplyr::left_join(diagnoses_df, by = "project_id") |>
dplyr::mutate(
# create a label for plotting
project_label = glue::glue("{project_id}:{diagnosis}")
)
# Turn into a dataframe with one row per library per celltype
# Make a table for each consensus method separately and then stack by consensus cell type old/new
# one row for old and one row for each cell type
# first add columns for total cells per library, number of new and old cell types
grouped_df <- all_results_df |>
dplyr::group_by(library_id) |>
dplyr::mutate(
total_cells_per_library = dplyr::n(),
num_old_celltypes = length(unique(consensus_annotation)),
num_new_celltypes = length(unique(existing_consensus_celltype_annotation))
) |>
dplyr::ungroup()
# get stats for old consensus cell types
old_consensus_df <- grouped_df |>
dplyr::group_by(project_id, project_label, diagnosis, library_id, sample_type, existing_consensus_celltype_annotation, existing_consensus_celltype_ontology) |>
dplyr::summarize(num_celltypes = unique(num_new_celltypes), # constant for each library
total_cells_per_annotation = dplyr::n(),
total_cells_per_library = unique(total_cells_per_library)) |>
dplyr::mutate(
# add percentage
percent_cells_annotation = round((total_cells_per_annotation / total_cells_per_library) * 100, 2)
) |>
dplyr::ungroup()
# stats for new consensus cell types
new_consensus_df <- grouped_df |>
dplyr::group_by(project_id, project_label, diagnosis, library_id, sample_type, consensus_annotation, consensus_ontology) |>
dplyr::summarize(num_celltypes = unique(num_old_celltypes), # constant for each library
total_cells_per_annotation = dplyr::n(),
total_cells_per_library = unique(total_cells_per_library)) |>
dplyr::mutate(
# add percentage
percent_cells_annotation = round((total_cells_per_annotation / total_cells_per_library) * 100, 2)
) |>
dplyr::ungroup()
# combine into a single df and add a column to indiciate old/new
df_list <- list(old_consensus_df, new_consensus_df) |>
purrr::set_names(c("no_scimilarity", "with_scimilarity"))
all_grouped_df <- dplyr::bind_rows(df_list, .id = "consensus_type")
Number of unknown cells
The plot below shows the number of cells annotated as unknown for
each library using old (without SCimilarity) vs. new (with
SCimilarity) consensus annotations. The red bar indicates
the median percentage of cells.

As expected, the percentage of cells annotated as unknown is lower
when SCimilarity is incorporated, suggesting we are able to
annotate more cells with a meaningful label that is not unknown.
Number of cell types observed
Below we look at the number of cell types observed in each project
for all samples.

As expected, we see that the number of cell types identified
increases with the addition of SCimilarity.
Heatmaps comparing old and new consensus cell types
In this section, we compare the top 15 cell type annotations for
consensus cell types with SCimilarity (rows) to consensus
cell types without SCimilarity (columns). All other cell
types not in the top 15 represented cell types in a project are grouped
into the “All remaining cell types” category.
Note that in some cases, there are fewer than 15 unique cell
types.
all_results_df <- all_results_df |>
dplyr::group_by(project_id) |>
dplyr::mutate(
# get most frequently observed cell types across libraries in that project
existing_consensus_celltype_lumped = forcats::fct_lump_n(existing_consensus_celltype_annotation, 15, other_level = "All remaining cell types", ties.method = "first") |>
# sort by frequency
forcats::fct_infreq() |>
# make sure all remaining and unknown are last, use this to assign colors in specific order
forcats::fct_relevel("All remaining cell types", after = Inf) |>
as.character(),
# do the same thing for the new consensus cell types
consensus_celltype_lumped = forcats::fct_lump_n(consensus_annotation, 15, other_level = "All remaining cell types", ties.method = "first") |>
# sort by frequency
forcats::fct_infreq() |>
# make sure all remaining and unknown are last, use this to assign colors in specific order
forcats::fct_relevel("All remaining cell types", after = Inf) |>
as.character(),
# jaccard functions expect a cell id column
cell_id = glue::glue("{library_id}-{barcodes}")
)
project_labels <- unique(all_results_df$project_label)
project_labels |>
purrr::walk(\(label){
all_results_df |>
dplyr::filter(project_label == label) |>
make_jaccard_heatmap(
annotation_col1 = "existing_consensus_celltype_lumped",
annotation_col2 = "consensus_celltype_lumped",
label1 = glue::glue("{label} \nNo Scimilarity"),
label2 = "With SCimilarity"
)
})





















In general we see a fair amount of agreement in the overall cell
types are defined with a bit more granularity in the cell types assigned
with SCimilarity.
Distribution of new consensus cell types
Now we look at the distribution of the cell types in each sample. For
these plots, we will pull out the top 9 cell types for each project. All
other cells will be labeled with “All remaining cell types”.
The top cell types are determined by counting how many libraries each
cell type is found in within a project and taking the most frequent
types.
plot_df <- all_grouped_df |>
dplyr::group_by(project_id) |>
# remove the old results from plotting
dplyr::select(-c(existing_consensus_celltype_annotation, existing_consensus_celltype_ontology)) |>
tidyr::drop_na(consensus_annotation) |>
dplyr::mutate(
# get most frequently observed cell types across libraries in that project
top_celltypes = forcats::fct_lump_n(consensus_annotation, 9, other_level = "All remaining cell types", ties.method = "first") |>
# sort by frequency
forcats::fct_infreq() |>
# make sure all remaining and unknown are last, use this to assign colors in specific order
forcats::fct_relevel("All remaining cell types", "Unknown", after = Inf)
) |>
dplyr::ungroup()
# get all unique cell types ordered by frequency
unique_celltypes <- plot_df |>
dplyr::filter(!top_celltypes %in% c("All remaining cell types", "Unknown")) |>
dplyr::pull(top_celltypes) |>
unique() |>
sort() |>
as.character()
# get color palette
colors <- c(
colorRampPalette(RColorBrewer::brewer.pal(12, "Paired"))(length(unique_celltypes)),
"grey60", # all remaining
"grey95" # unknown
)
names(colors) <- c(unique_celltypes, "All remaining cell types", "Unknown")
# stacked bar chart showing the distribution of the top 9 cell types for each project, including Unknown
project_labels |>
purrr::map(\(label){
project_df <- plot_df |>
dplyr::filter(project_label == label) |>
dplyr::mutate(
# relevel factors for specific project
top_celltypes = forcats::fct_infreq(top_celltypes) |>
forcats::fct_relevel("All remaining cell types", "Unknown", after = Inf)
)
# make a stacked bar chart with top cell types
ggplot(project_df) +
aes(
x = library_id,
y = percent_cells_annotation,
fill = top_celltypes
) +
geom_col() +
# split samples based on sample type, patient tissue or pdx
facet_wrap(vars(sample_type), scales ="free") +
scale_y_continuous(expand = c(0,0)) +
scale_fill_manual(values = colors, name = "cell type") +
ggtitle(label) +
theme(axis.text.x = element_blank())
}) |>
patchwork::wrap_plots(ncol = 1)

We are definitely labeling more cells than we were previously and in
some cases we may be assigning a “normal” cell type to tumor cells. This
is not totally surprising as many solid tumors have cells that resemble
fibroblasts and muscle cells. We also see various progenitors and HSCs
being labeled in the leukemias which reflect how leukemia cells resemble
those cell types. Overall, I think adding in SCimilarity is
giving us a lot more information than without it.
Session info
# record the versions of the packages used in this analysis and other environment information
sessionInfo()
R version 4.4.2 (2024-10-31)
Platform: aarch64-apple-darwin20
Running under: macOS Sequoia 15.6.1
Matrix products: default
BLAS: /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib
LAPACK: /Library/Frameworks/R.framework/Versions/4.4-arm64/Resources/lib/libRlapack.dylib; LAPACK version 3.12.0
locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
time zone: America/Chicago
tzcode source: internal
attached base packages:
[1] stats graphics grDevices datasets utils methods base
other attached packages:
[1] ggplot2_3.5.1
loaded via a namespace (and not attached):
Error in x[["Version"]] : subscript out of bounds
LS0tCnRpdGxlOiAiQ29tcGFyZSBjb25zZW5zdXMgY2VsbCB0eXBlcyB3aXRoIFNDaW1pbGFyaXR5IHRvIGNvbnNlbnN1cyBjZWxsIHR5cGVzIHdpdGhvdXQgU0NpbWlsYXJpdHkiCmF1dGhvcjogQWxseSBIYXdraW5zCmRhdGU6ICJgciBTeXMuRGF0ZSgpYCIKb3V0cHV0OgogIGh0bWxfbm90ZWJvb2s6CiAgICB0b2M6IHRydWUKICAgIHRvY19kZXB0aDogMwogICAgY29kZV9mb2xkaW5nOiBoaWRlCi0tLQoKVGhpcyBub3RlYm9vayBjb21wYXJlcyB0aGUgcmVzdWx0cyBmcm9tIGFzc2lnbmluZyBjb25zZW5zdXMgY2VsbCB0eXBlcyB3aXRoIGBTQ2ltaWxhcml0eWAgdG8gdGhlIHByZXZpb3VzIGNvbnNlbnN1cyBjZWxsIHR5cGVzIHdpdGhvdXQgYFNDaW1pbGFyaXR5YC4gCkFsbCByZXN1bHRzIGZyb20gdGhlIGBjZWxsLXR5cGUtY29uc2Vuc3VzYCBtb2R1bGUgaW4gYE9wZW5TY1BDQS1uZmAgbXVzdCBiZSBzYXZlZCB0byBgcmVzdWx0c2AgcHJpb3IgdG8gcmVuZGVyaW5nIHRoaXMgbm90ZWJvb2suIAoKTm90ZSB0aGF0IGFueSBtdWx0aXBsZXhlZCBsaWJyYXJpZXMgaGF2ZSBiZWVuIGV4Y2x1ZGVkIGZyb20gdGhpcyBub3RlYm9vayBhcyBgU0NpbWlsYXJpdHlgIGlzIG5vdCBydW4gb24gdGhvc2UgbGlicmFyaWVzIHdoZW4gdXNpbmcgdGhlIGBPcGVuU2NQQ0EtbmZgIG1vZHVsZS4gCgpgYGB7ciBwYWNrYWdlc30Kc3VwcHJlc3NQYWNrYWdlU3RhcnR1cE1lc3NhZ2VzKHsKICAjIGxvYWQgcmVxdWlyZWQgcGFja2FnZXMKICBsaWJyYXJ5KGdncGxvdDIpCn0pCgojIFNldCBkZWZhdWx0IGdncGxvdCB0aGVtZQp0aGVtZV9zZXQoCiAgdGhlbWVfY2xhc3NpYygpCikKCiMgRGVmaW5lIGNvbG9yIHJhbXAgZm9yIHNoYXJlZCB1c2UgaW4gdGhlIGhlYXRtYXBzCmhlYXRtYXBfY29sX2Z1biA8LSBjaXJjbGl6ZTo6Y29sb3JSYW1wMihjKDAsIDEpLCBjb2xvcnMgPSBjKCJ3aGl0ZSIsICJkYXJrc2xhdGVibHVlIikpCiMgU2V0IGhlYXRtYXAgcGFkZGluZyBvcHRpb24KQ29tcGxleEhlYXRtYXA6Omh0X29wdChUSVRMRV9QQURESU5HID0gZ3JpZDo6dW5pdCgwLjYsICJpbiIpKQoKIyBzZXR0aW5ncwpvcHRpb25zKAogIGRwbHlyLnN1bW1hcmlzZS5pbmZvcm0gPSBGQUxTRSwgCiAgcmVhZHIuc2hvd19jb2xfdHlwZXMgPSBGQUxTRQopCmBgYAoKCiMjIERhdGEgc2V0dXAKCgpgYGB7ciBiYXNlIHBhdGhzfQojIFRoZSBiYXNlIHBhdGggZm9yIHRoZSBPcGVuU2NQQ0EgcmVwb3NpdG9yeSwgZm91bmQgYnkgaXRzIChoaWRkZW4pIC5naXQgZGlyZWN0b3J5CnJlcG9zaXRvcnlfYmFzZSA8LSBycHJvanJvb3Q6OmZpbmRfcm9vdChycHJvanJvb3Q6OmlzX2dpdF9yb290KQptb2R1bGVfYmFzZSA8LSBmaWxlLnBhdGgocmVwb3NpdG9yeV9iYXNlLCAiYW5hbHlzZXMiLCAiY2VsbC10eXBlLWNvbnNlbnN1cyIpCgojIHJlc3VsdHMgZGlyZWN0b3J5IHdpdGggY2VsbC10eXBlLWNvbnNlbnN1cyAKcmVzdWx0c19kaXIgPC0gZmlsZS5wYXRoKG1vZHVsZV9iYXNlLCAicmVzdWx0cyIsICJjZWxsLXR5cGUtY29uc2Vuc3VzIikKCiMgZGlhZ25vc2VzIHRhYmxlIHVzZWQgZm9yIGxhYmVsaW5nIHBsb3RzIApkaWFnbm9zZXNfZmlsZSA8LSBmaWxlLnBhdGgobW9kdWxlX2Jhc2UsICJzYW1wbGUtaW5mbyIsICJwcm9qZWN0LWRpYWdub3Nlcy50c3YiKQpgYGAKCmBgYHtyfQojIHVzZSB0aGUgamFjY2FyZCBmdW5jdGlvbnMgZnJvbSBjZWxsLXR5cGUtbmV1cm9ibGFzdG9tYS0wNCBtb2R1bGUKamFjY2FyZF9mdW5jdGlvbnMgPC0gZmlsZS5wYXRoKHJlcG9zaXRvcnlfYmFzZSwgImFuYWx5c2VzIiwgImNlbGwtdHlwZS1uZXVyb2JsYXN0b21hLTA0IiwgInNjcmlwdHMiLCAidXRpbHMiLCAiamFjY2FyZC11dGlscy5SIikKc291cmNlKGphY2NhcmRfZnVuY3Rpb25zKQpgYGAKCmBgYHtyfQojIGxpc3QgYWxsIHJlc3VsdHMgZmlsZXMgCnJlc3VsdHNfZmlsZXMgPC0gbGlzdC5maWxlcyhyZXN1bHRzX2RpciwgcGF0dGVybiA9ICJfY29uc2Vuc3VzLWNlbGwtdHlwZXNcXC50c3YuXFxneiQiLCByZWN1cnNpdmUgPSBUUlVFLCBmdWxsLm5hbWVzID0gVFJVRSkKCiMgZGVmaW5lIGNlbGwgbGluZSBwcm9qZWN0cyB0byByZW1vdmUKY2VsbF9saW5lX3Byb2plY3RzIDwtIGMoIlNDUENQMDAwMDIwIiwgIlNDUENQMDAwMDI0IikKYGBgCgoKYGBge3IsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CiMgcmVhZCBpbiBkaWFnbm9zZXMKZGlhZ25vc2VzX2RmIDwtIHJlYWRyOjpyZWFkX3RzdihkaWFnbm9zZXNfZmlsZSkKCiMgcmVhZCBpbiByZXN1bHRzIGFuZCBwcmVwIGRhdGEgZnJhbWUgZm9yIHBsb3R0aW5nIAphbGxfcmVzdWx0c19kZiA8LSByZXN1bHRzX2ZpbGVzIHw+IAogIHB1cnJyOjptYXAocmVhZHI6OnJlYWRfdHN2LCBjb2xfdHlwZXMgPSAiYyIpfD4gCiAgZHBseXI6OmJpbmRfcm93cygpIHw+IAogICMgcmVtb3ZlIGNlbGwgbGluZSBwcm9qZWN0cwogIGRwbHlyOjpmaWx0ZXIoIXByb2plY3RfaWQgJWluJSBjZWxsX2xpbmVfcHJvamVjdHMsCiAgICAgICAgICAgICAgICAjIHJlbW92ZSBtdWx0aXBsZXhlZCBsaWJyYXJpZXMgc2luY2Ugd2UgZGlkbid0IHJ1biBzY2ltaWxhcml0eSBvbiB0aGVtCiAgICAgICAgICAgICAgICAhc3RyaW5ncjo6c3RyX2RldGVjdChzYW1wbGVfaWQsICI7IikpIHw+IAogICMgYWRkIGluIGRpYWdub3NlcyAKICBkcGx5cjo6bGVmdF9qb2luKGRpYWdub3Nlc19kZiwgYnkgPSAicHJvamVjdF9pZCIpIHw+IAogIGRwbHlyOjptdXRhdGUoCiAgICAjIGNyZWF0ZSBhIGxhYmVsIGZvciBwbG90dGluZwogICAgcHJvamVjdF9sYWJlbCA9IGdsdWU6OmdsdWUoIntwcm9qZWN0X2lkfTp7ZGlhZ25vc2lzfSIpCiAgKQpgYGAKCgpgYGB7ciwgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0KIyBUdXJuIGludG8gYSBkYXRhZnJhbWUgd2l0aCBvbmUgcm93IHBlciBsaWJyYXJ5IHBlciBjZWxsdHlwZQojIE1ha2UgYSB0YWJsZSBmb3IgZWFjaCBjb25zZW5zdXMgbWV0aG9kIHNlcGFyYXRlbHkgYW5kIHRoZW4gc3RhY2sgYnkgY29uc2Vuc3VzIGNlbGwgdHlwZSBvbGQvbmV3IAojIG9uZSByb3cgZm9yIG9sZCBhbmQgb25lIHJvdyBmb3IgZWFjaCBjZWxsIHR5cGUKCiMgZmlyc3QgYWRkIGNvbHVtbnMgZm9yIHRvdGFsIGNlbGxzIHBlciBsaWJyYXJ5LCBudW1iZXIgb2YgbmV3IGFuZCBvbGQgY2VsbCB0eXBlcwpncm91cGVkX2RmIDwtIGFsbF9yZXN1bHRzX2RmIHw+IAogICAgZHBseXI6Omdyb3VwX2J5KGxpYnJhcnlfaWQpIHw+IAogICAgZHBseXI6Om11dGF0ZSgKICAgICAgdG90YWxfY2VsbHNfcGVyX2xpYnJhcnkgPSBkcGx5cjo6bigpLAogICAgICBudW1fb2xkX2NlbGx0eXBlcyA9IGxlbmd0aCh1bmlxdWUoY29uc2Vuc3VzX2Fubm90YXRpb24pKSwKICAgICAgbnVtX25ld19jZWxsdHlwZXMgPSBsZW5ndGgodW5pcXVlKGV4aXN0aW5nX2NvbnNlbnN1c19jZWxsdHlwZV9hbm5vdGF0aW9uKSkKICAgICkgfD4KICAgIGRwbHlyOjp1bmdyb3VwKCkKICAKIyBnZXQgc3RhdHMgZm9yIG9sZCBjb25zZW5zdXMgY2VsbCB0eXBlcwpvbGRfY29uc2Vuc3VzX2RmIDwtIGdyb3VwZWRfZGYgfD4gCiAgICBkcGx5cjo6Z3JvdXBfYnkocHJvamVjdF9pZCwgcHJvamVjdF9sYWJlbCwgZGlhZ25vc2lzLCBsaWJyYXJ5X2lkLCBzYW1wbGVfdHlwZSwgZXhpc3RpbmdfY29uc2Vuc3VzX2NlbGx0eXBlX2Fubm90YXRpb24sIGV4aXN0aW5nX2NvbnNlbnN1c19jZWxsdHlwZV9vbnRvbG9neSkgfD4gCiAgICBkcGx5cjo6c3VtbWFyaXplKG51bV9jZWxsdHlwZXMgPSB1bmlxdWUobnVtX25ld19jZWxsdHlwZXMpLCAjIGNvbnN0YW50IGZvciBlYWNoIGxpYnJhcnkKICAgICAgICAgICAgICAgICAgICAgdG90YWxfY2VsbHNfcGVyX2Fubm90YXRpb24gPSBkcGx5cjo6bigpLAogICAgICAgICAgICAgICAgICAgICB0b3RhbF9jZWxsc19wZXJfbGlicmFyeSA9IHVuaXF1ZSh0b3RhbF9jZWxsc19wZXJfbGlicmFyeSkpIHw+CiAgICBkcGx5cjo6bXV0YXRlKAogICAgICAjIGFkZCBwZXJjZW50YWdlIAogICAgICBwZXJjZW50X2NlbGxzX2Fubm90YXRpb24gPSByb3VuZCgodG90YWxfY2VsbHNfcGVyX2Fubm90YXRpb24gLyB0b3RhbF9jZWxsc19wZXJfbGlicmFyeSkgKiAxMDAsIDIpCiAgICApIHw+IAogICAgZHBseXI6OnVuZ3JvdXAoKQoKIyBzdGF0cyBmb3IgbmV3IGNvbnNlbnN1cyBjZWxsIHR5cGVzCm5ld19jb25zZW5zdXNfZGYgPC0gZ3JvdXBlZF9kZiB8PiAKICAgIGRwbHlyOjpncm91cF9ieShwcm9qZWN0X2lkLCBwcm9qZWN0X2xhYmVsLCBkaWFnbm9zaXMsIGxpYnJhcnlfaWQsIHNhbXBsZV90eXBlLCBjb25zZW5zdXNfYW5ub3RhdGlvbiwgY29uc2Vuc3VzX29udG9sb2d5KSB8PiAKICAgIGRwbHlyOjpzdW1tYXJpemUobnVtX2NlbGx0eXBlcyA9IHVuaXF1ZShudW1fb2xkX2NlbGx0eXBlcyksICMgY29uc3RhbnQgZm9yIGVhY2ggbGlicmFyeQogICAgICAgICAgICAgICAgICAgICB0b3RhbF9jZWxsc19wZXJfYW5ub3RhdGlvbiA9IGRwbHlyOjpuKCksCiAgICAgICAgICAgICAgICAgICAgIHRvdGFsX2NlbGxzX3Blcl9saWJyYXJ5ID0gdW5pcXVlKHRvdGFsX2NlbGxzX3Blcl9saWJyYXJ5KSkgfD4KICAgIGRwbHlyOjptdXRhdGUoCiAgICAgICMgYWRkIHBlcmNlbnRhZ2UgCiAgICAgIHBlcmNlbnRfY2VsbHNfYW5ub3RhdGlvbiA9IHJvdW5kKCh0b3RhbF9jZWxsc19wZXJfYW5ub3RhdGlvbiAvIHRvdGFsX2NlbGxzX3Blcl9saWJyYXJ5KSAqIDEwMCwgMikKICAgICkgfD4gCiAgICBkcGx5cjo6dW5ncm91cCgpCgojIGNvbWJpbmUgaW50byBhIHNpbmdsZSBkZiBhbmQgYWRkIGEgY29sdW1uIHRvIGluZGljaWF0ZSBvbGQvbmV3IApkZl9saXN0IDwtIGxpc3Qob2xkX2NvbnNlbnN1c19kZiwgbmV3X2NvbnNlbnN1c19kZikgfD4gCiAgcHVycnI6OnNldF9uYW1lcyhjKCJub19zY2ltaWxhcml0eSIsICJ3aXRoX3NjaW1pbGFyaXR5IikpCgphbGxfZ3JvdXBlZF9kZiA8LSBkcGx5cjo6YmluZF9yb3dzKGRmX2xpc3QsIC5pZCA9ICJjb25zZW5zdXNfdHlwZSIpCmBgYAoKIyMgTnVtYmVyIG9mIHVua25vd24gY2VsbHMKClRoZSBwbG90IGJlbG93IHNob3dzIHRoZSBudW1iZXIgb2YgY2VsbHMgYW5ub3RhdGVkIGFzIHVua25vd24gZm9yIGVhY2ggbGlicmFyeSB1c2luZyBvbGQgKHdpdGhvdXQgYFNDaW1pbGFyaXR5YCkgdnMuIG5ldyAod2l0aCBgU0NpbWlsYXJpdHlgKSBjb25zZW5zdXMgYW5ub3RhdGlvbnMuIApUaGUgcmVkIGJhciBpbmRpY2F0ZXMgdGhlIG1lZGlhbiBwZXJjZW50YWdlIG9mIGNlbGxzLiAKCmBgYHtyLCBmaWcuaGVpZ2h0PTd9CiMgb25seSBnZXQgdGhlIHVua25vd24gY2VsbHMKdW5rbm93bl9vbmx5IDwtIGFsbF9ncm91cGVkX2RmIHw+IAogIGRwbHlyOjpmaWx0ZXIoY29uc2Vuc3VzX2Fubm90YXRpb24gPT0gIlVua25vd24iIHwgZXhpc3RpbmdfY29uc2Vuc3VzX2NlbGx0eXBlX2Fubm90YXRpb24gPT0gIlVua25vd24iKQoKIyBjb21wYXJlIHRoZSBkaXN0cmlidXRpb24gb2YgdW5rbm93biB3aXRoIGFuZCB3aXRob3V0IHNjaW1pbGFyaXR5CmdncGxvdCh1bmtub3duX29ubHksIGFlcyh4ID0gY29uc2Vuc3VzX3R5cGUsIHkgPSBwZXJjZW50X2NlbGxzX2Fubm90YXRpb24pKSArCiAgZ2dmb3JjZTo6Z2VvbV9zaW5hKHNpemUgPSAwLjEpICsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDkwLCBoanVzdCA9IDEsIHZqdXN0ID0gMC41KSwKICAgICAgICBwbG90Lm1hcmdpbiA9IG1hcmdpbigxMCwxMCwxMCwxMCkpICsKICBsYWJzKAogICAgeCA9ICIiLCAKICAgIHkgPSAiUGVyY2VudCBvZiBjZWxscyBhbm5vdGF0ZWQgYXMgVW5rbm93biIKICApICsKICBzdGF0X3N1bW1hcnkoZnVuPW1lZGlhbiwgZ2VvbT0iY3Jvc3NiYXIiICwgY29sb3IgPSAicmVkIiwgbGluZXdpZHRoID0gMC4yKQogIApgYGAKCkFzIGV4cGVjdGVkLCB0aGUgcGVyY2VudGFnZSBvZiBjZWxscyBhbm5vdGF0ZWQgYXMgdW5rbm93biBpcyBsb3dlciB3aGVuIGBTQ2ltaWxhcml0eWAgaXMgaW5jb3Jwb3JhdGVkLCBzdWdnZXN0aW5nIHdlIGFyZSBhYmxlIHRvIGFubm90YXRlIG1vcmUgY2VsbHMgd2l0aCBhIG1lYW5pbmdmdWwgbGFiZWwgdGhhdCBpcyBub3QgdW5rbm93bi4gCgojIyBOdW1iZXIgb2YgY2VsbCB0eXBlcyBvYnNlcnZlZAoKQmVsb3cgd2UgbG9vayBhdCB0aGUgbnVtYmVyIG9mIGNlbGwgdHlwZXMgb2JzZXJ2ZWQgaW4gZWFjaCBwcm9qZWN0IGZvciBhbGwgc2FtcGxlcy4gCgpgYGB7ciwgZmlnLmhlaWdodD0xNSwgZmlnLndpZHRoPTEwfQpudW1fY2VsbHR5cGVzX2RmIDwtIGFsbF9ncm91cGVkX2RmIHw+IAogICMgYWRkIGEgbmV3IGxpbmUgZm9yIGZhY2V0IGxhYmVscyAKICBkcGx5cjo6bXV0YXRlKGZhY2V0X2xhYmVsID0gZ2x1ZTo6Z2x1ZSgie3Byb2plY3RfaWR9XG57ZGlhZ25vc2lzfSIpKSB8PgogIGRwbHlyOjpzZWxlY3QoZmFjZXRfbGFiZWwsIGxpYnJhcnlfaWQsIG51bV9jZWxsdHlwZXMsIGNvbnNlbnN1c190eXBlKSB8PiAKICB1bmlxdWUoKQoKZ2dwbG90KG51bV9jZWxsdHlwZXNfZGYsIAogICAgICAgYWVzKHggPSBjb25zZW5zdXNfdHlwZSwgeSA9IG51bV9jZWxsdHlwZXMsIGdyb3VwID0gbGlicmFyeV9pZCkpICsKICBnZW9tX3BvaW50KGFlcyhjb2xvciA9IGNvbnNlbnN1c190eXBlKSkgKwogIGdlb21fbGluZShhZXMoZ3JvdXAgPSBsaWJyYXJ5X2lkKSwgY29sb3IgPSAiZ3JheTYwIiwgYWxwaGEgPSAwLjcpICsKICBmYWNldF93cmFwKHZhcnMoZmFjZXRfbGFiZWwpLCBuY29sID0gMykgKwogIGxhYnMoCiAgICB4ID0gIkNvbnNlbnN1cyBjZWxsIHR5cGUiLAogICAgeSA9ICJOdW1iZXIgb2YgY2VsbCB0eXBlcyIKICApICsKICB0aGVtZV9idygpCgpgYGAKCkFzIGV4cGVjdGVkLCB3ZSBzZWUgdGhhdCB0aGUgbnVtYmVyIG9mIGNlbGwgdHlwZXMgaWRlbnRpZmllZCBpbmNyZWFzZXMgd2l0aCB0aGUgYWRkaXRpb24gb2YgYFNDaW1pbGFyaXR5YC4gCgojIyBIZWF0bWFwcyBjb21wYXJpbmcgb2xkIGFuZCBuZXcgY29uc2Vuc3VzIGNlbGwgdHlwZXMKCkluIHRoaXMgc2VjdGlvbiwgd2UgY29tcGFyZSB0aGUgdG9wIDE1IGNlbGwgdHlwZSBhbm5vdGF0aW9ucyBmb3IgY29uc2Vuc3VzIGNlbGwgdHlwZXMgd2l0aCBgU0NpbWlsYXJpdHlgIChyb3dzKSB0byBjb25zZW5zdXMgY2VsbCB0eXBlcyB3aXRob3V0IGBTQ2ltaWxhcml0eWAgKGNvbHVtbnMpLiAKQWxsIG90aGVyIGNlbGwgdHlwZXMgbm90IGluIHRoZSB0b3AgMTUgcmVwcmVzZW50ZWQgY2VsbCB0eXBlcyBpbiBhIHByb2plY3QgYXJlIGdyb3VwZWQgaW50byB0aGUgIkFsbCByZW1haW5pbmcgY2VsbCB0eXBlcyIgY2F0ZWdvcnkuIAoKTm90ZSB0aGF0IGluIHNvbWUgY2FzZXMsIHRoZXJlIGFyZSBmZXdlciB0aGFuIDE1IHVuaXF1ZSBjZWxsIHR5cGVzLiAKCmBgYHtyLCB3YXJuaW5nPUZBTFNFfQphbGxfcmVzdWx0c19kZiA8LSBhbGxfcmVzdWx0c19kZiB8PiAKICAgIGRwbHlyOjpncm91cF9ieShwcm9qZWN0X2lkKSB8PiAKICAgIGRwbHlyOjptdXRhdGUoCiAgICAgICMgZ2V0IG1vc3QgZnJlcXVlbnRseSBvYnNlcnZlZCBjZWxsIHR5cGVzIGFjcm9zcyBsaWJyYXJpZXMgaW4gdGhhdCBwcm9qZWN0IAogICAgICBleGlzdGluZ19jb25zZW5zdXNfY2VsbHR5cGVfbHVtcGVkID0gZm9yY2F0czo6ZmN0X2x1bXBfbihleGlzdGluZ19jb25zZW5zdXNfY2VsbHR5cGVfYW5ub3RhdGlvbiwgMTUsIG90aGVyX2xldmVsID0gIkFsbCByZW1haW5pbmcgY2VsbCB0eXBlcyIsIHRpZXMubWV0aG9kID0gImZpcnN0IikgfD4gCiAgICAgICAgIyBzb3J0IGJ5IGZyZXF1ZW5jeSAKICAgICAgICBmb3JjYXRzOjpmY3RfaW5mcmVxKCkgfD4gCiAgICAgICAgIyBtYWtlIHN1cmUgYWxsIHJlbWFpbmluZyBhbmQgdW5rbm93biBhcmUgbGFzdCwgdXNlIHRoaXMgdG8gYXNzaWduIGNvbG9ycyBpbiBzcGVjaWZpYyBvcmRlcgogICAgICAgIGZvcmNhdHM6OmZjdF9yZWxldmVsKCJBbGwgcmVtYWluaW5nIGNlbGwgdHlwZXMiLCBhZnRlciA9IEluZikgfD4gCiAgICAgICAgYXMuY2hhcmFjdGVyKCksIAogICAgICAjIGRvIHRoZSBzYW1lIHRoaW5nIGZvciB0aGUgbmV3IGNvbnNlbnN1cyBjZWxsIHR5cGVzCiAgICAgIGNvbnNlbnN1c19jZWxsdHlwZV9sdW1wZWQgPSBmb3JjYXRzOjpmY3RfbHVtcF9uKGNvbnNlbnN1c19hbm5vdGF0aW9uLCAxNSwgb3RoZXJfbGV2ZWwgPSAiQWxsIHJlbWFpbmluZyBjZWxsIHR5cGVzIiwgdGllcy5tZXRob2QgPSAiZmlyc3QiKSB8PiAKICAgICAgICAjIHNvcnQgYnkgZnJlcXVlbmN5IAogICAgICAgIGZvcmNhdHM6OmZjdF9pbmZyZXEoKSB8PiAKICAgICAgICAjIG1ha2Ugc3VyZSBhbGwgcmVtYWluaW5nIGFuZCB1bmtub3duIGFyZSBsYXN0LCB1c2UgdGhpcyB0byBhc3NpZ24gY29sb3JzIGluIHNwZWNpZmljIG9yZGVyCiAgICAgICAgZm9yY2F0czo6ZmN0X3JlbGV2ZWwoIkFsbCByZW1haW5pbmcgY2VsbCB0eXBlcyIsIGFmdGVyID0gSW5mKSB8PiAKICAgICAgICBhcy5jaGFyYWN0ZXIoKSwgCiAgICAgICMgamFjY2FyZCBmdW5jdGlvbnMgZXhwZWN0IGEgY2VsbCBpZCBjb2x1bW4KICAgICAgY2VsbF9pZCA9IGdsdWU6OmdsdWUoIntsaWJyYXJ5X2lkfS17YmFyY29kZXN9IikKICAgICkKCnByb2plY3RfbGFiZWxzIDwtIHVuaXF1ZShhbGxfcmVzdWx0c19kZiRwcm9qZWN0X2xhYmVsKQpgYGAKCgpgYGB7ciwgZmlnLmhlaWdodCA9IDEwLCBmaWcud2lkdGggPSAxMH0KcHJvamVjdF9sYWJlbHMgfD4gCiAgcHVycnI6OndhbGsoXChsYWJlbCl7CiAgICAKICAgIGFsbF9yZXN1bHRzX2RmIHw+IAogICAgICBkcGx5cjo6ZmlsdGVyKHByb2plY3RfbGFiZWwgPT0gbGFiZWwpIHw+IAogICAgICBtYWtlX2phY2NhcmRfaGVhdG1hcCgKICAgICAgICBhbm5vdGF0aW9uX2NvbDEgPSAiZXhpc3RpbmdfY29uc2Vuc3VzX2NlbGx0eXBlX2x1bXBlZCIsCiAgICAgICAgYW5ub3RhdGlvbl9jb2wyID0gImNvbnNlbnN1c19jZWxsdHlwZV9sdW1wZWQiLAogICAgICAgIGxhYmVsMSA9IGdsdWU6OmdsdWUoIntsYWJlbH0gXG5ObyBTY2ltaWxhcml0eSIpLAogICAgICAgIGxhYmVsMiA9ICJXaXRoIFNDaW1pbGFyaXR5IgogICAgICApCiAgICAKICB9KQpgYGAKCgpJbiBnZW5lcmFsIHdlIHNlZSBhIGZhaXIgYW1vdW50IG9mIGFncmVlbWVudCBpbiB0aGUgb3ZlcmFsbCBjZWxsIHR5cGVzIGFyZSBkZWZpbmVkIHdpdGggYSBiaXQgbW9yZSBncmFudWxhcml0eSBpbiB0aGUgY2VsbCB0eXBlcyBhc3NpZ25lZCB3aXRoIGBTQ2ltaWxhcml0eWAuIAoKIyMgRGlzdHJpYnV0aW9uIG9mIG5ldyBjb25zZW5zdXMgY2VsbCB0eXBlcyAKCk5vdyB3ZSBsb29rIGF0IHRoZSBkaXN0cmlidXRpb24gb2YgdGhlIGNlbGwgdHlwZXMgaW4gZWFjaCBzYW1wbGUuIApGb3IgdGhlc2UgcGxvdHMsIHdlIHdpbGwgcHVsbCBvdXQgdGhlIHRvcCA5IGNlbGwgdHlwZXMgZm9yIGVhY2ggcHJvamVjdC4gCkFsbCBvdGhlciBjZWxscyB3aWxsIGJlIGxhYmVsZWQgd2l0aCAiQWxsIHJlbWFpbmluZyBjZWxsIHR5cGVzIi4gCgpUaGUgdG9wIGNlbGwgdHlwZXMgYXJlIGRldGVybWluZWQgYnkgY291bnRpbmcgaG93IG1hbnkgbGlicmFyaWVzIGVhY2ggY2VsbCB0eXBlIGlzIGZvdW5kIGluIHdpdGhpbiBhIHByb2plY3QgYW5kIHRha2luZyB0aGUgbW9zdCBmcmVxdWVudCB0eXBlcy4gCgpgYGB7ciwgd2FybmluZz1GQUxTRX0KcGxvdF9kZiA8LSBhbGxfZ3JvdXBlZF9kZiB8PiAKICBkcGx5cjo6Z3JvdXBfYnkocHJvamVjdF9pZCkgfD4gCiAgIyByZW1vdmUgdGhlIG9sZCByZXN1bHRzIGZyb20gcGxvdHRpbmcKICBkcGx5cjo6c2VsZWN0KC1jKGV4aXN0aW5nX2NvbnNlbnN1c19jZWxsdHlwZV9hbm5vdGF0aW9uLCBleGlzdGluZ19jb25zZW5zdXNfY2VsbHR5cGVfb250b2xvZ3kpKSB8PiAKICB0aWR5cjo6ZHJvcF9uYShjb25zZW5zdXNfYW5ub3RhdGlvbikgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgICMgZ2V0IG1vc3QgZnJlcXVlbnRseSBvYnNlcnZlZCBjZWxsIHR5cGVzIGFjcm9zcyBsaWJyYXJpZXMgaW4gdGhhdCBwcm9qZWN0IAogICAgdG9wX2NlbGx0eXBlcyA9IGZvcmNhdHM6OmZjdF9sdW1wX24oY29uc2Vuc3VzX2Fubm90YXRpb24sIDksIG90aGVyX2xldmVsID0gIkFsbCByZW1haW5pbmcgY2VsbCB0eXBlcyIsIHRpZXMubWV0aG9kID0gImZpcnN0IikgfD4gCiAgICAgICMgc29ydCBieSBmcmVxdWVuY3kgCiAgICAgIGZvcmNhdHM6OmZjdF9pbmZyZXEoKSB8PiAKICAgICAgIyBtYWtlIHN1cmUgYWxsIHJlbWFpbmluZyBhbmQgdW5rbm93biBhcmUgbGFzdCwgdXNlIHRoaXMgdG8gYXNzaWduIGNvbG9ycyBpbiBzcGVjaWZpYyBvcmRlcgogICAgICBmb3JjYXRzOjpmY3RfcmVsZXZlbCgiQWxsIHJlbWFpbmluZyBjZWxsIHR5cGVzIiwgIlVua25vd24iLCBhZnRlciA9IEluZikKICApIHw+IAogIGRwbHlyOjp1bmdyb3VwKCkKCiMgZ2V0IGFsbCB1bmlxdWUgY2VsbCB0eXBlcyBvcmRlcmVkIGJ5IGZyZXF1ZW5jeSAKdW5pcXVlX2NlbGx0eXBlcyA8LSBwbG90X2RmIHw+IAogIGRwbHlyOjpmaWx0ZXIoIXRvcF9jZWxsdHlwZXMgJWluJSBjKCJBbGwgcmVtYWluaW5nIGNlbGwgdHlwZXMiLCAiVW5rbm93biIpKSB8PiAKICBkcGx5cjo6cHVsbCh0b3BfY2VsbHR5cGVzKSB8PiAKICB1bmlxdWUoKSB8PgogIHNvcnQoKSB8PiAKICBhcy5jaGFyYWN0ZXIoKQoKIyBnZXQgY29sb3IgcGFsZXR0ZQpjb2xvcnMgPC0gYygKICBjb2xvclJhbXBQYWxldHRlKFJDb2xvckJyZXdlcjo6YnJld2VyLnBhbCgxMiwgIlBhaXJlZCIpKShsZW5ndGgodW5pcXVlX2NlbGx0eXBlcykpLAogICJncmV5NjAiLCAjIGFsbCByZW1haW5pbmcKICAiZ3JleTk1IiAjIHVua25vd24KKQpuYW1lcyhjb2xvcnMpIDwtIGModW5pcXVlX2NlbGx0eXBlcywgIkFsbCByZW1haW5pbmcgY2VsbCB0eXBlcyIsICJVbmtub3duIikKYGBgCgoKYGBge3IsIGZpZy5oZWlnaHQ9NjAsIGZpZy53aWR0aD0xMH0KIyBzdGFja2VkIGJhciBjaGFydCBzaG93aW5nIHRoZSBkaXN0cmlidXRpb24gb2YgdGhlIHRvcCA5IGNlbGwgdHlwZXMgZm9yIGVhY2ggcHJvamVjdCwgaW5jbHVkaW5nIFVua25vd24KcHJvamVjdF9sYWJlbHMgfD4gCiAgcHVycnI6Om1hcChcKGxhYmVsKXsKICAgIAogICAgcHJvamVjdF9kZiA8LSBwbG90X2RmIHw+IAogICAgICBkcGx5cjo6ZmlsdGVyKHByb2plY3RfbGFiZWwgPT0gbGFiZWwpIHw+IAogICAgICBkcGx5cjo6bXV0YXRlKAogICAgICAgICMgcmVsZXZlbCBmYWN0b3JzIGZvciBzcGVjaWZpYyBwcm9qZWN0IAogICAgICAgIHRvcF9jZWxsdHlwZXMgPSBmb3JjYXRzOjpmY3RfaW5mcmVxKHRvcF9jZWxsdHlwZXMpIHw+IAogICAgICAgICAgZm9yY2F0czo6ZmN0X3JlbGV2ZWwoIkFsbCByZW1haW5pbmcgY2VsbCB0eXBlcyIsICJVbmtub3duIiwgYWZ0ZXIgPSBJbmYpCiAgICAgICkKICAgIAogICAgIyBtYWtlIGEgc3RhY2tlZCBiYXIgY2hhcnQgd2l0aCB0b3AgY2VsbCB0eXBlcyAKICAgIGdncGxvdChwcm9qZWN0X2RmKSArIAogICAgICBhZXMoCiAgICAgICAgeCA9IGxpYnJhcnlfaWQsIAogICAgICAgIHkgPSBwZXJjZW50X2NlbGxzX2Fubm90YXRpb24sIAogICAgICAgIGZpbGwgPSB0b3BfY2VsbHR5cGVzCiAgICAgICkgKwogICAgICBnZW9tX2NvbCgpICsgCiAgICAgICMgc3BsaXQgc2FtcGxlcyBiYXNlZCBvbiBzYW1wbGUgdHlwZSwgcGF0aWVudCB0aXNzdWUgb3IgcGR4IAogICAgICBmYWNldF93cmFwKHZhcnMoc2FtcGxlX3R5cGUpLCBzY2FsZXMgPSJmcmVlIikgKwogICAgICBzY2FsZV95X2NvbnRpbnVvdXMoZXhwYW5kID0gYygwLDApKSArCiAgICAgIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcyA9IGNvbG9ycywgbmFtZSA9ICJjZWxsIHR5cGUiKSArCiAgICAgIGdndGl0bGUobGFiZWwpICsKICAgICAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X2JsYW5rKCkpCiAgCiAgICB9KSB8PgogIHBhdGNod29yazo6d3JhcF9wbG90cyhuY29sID0gMSkKYGBgCgoKV2UgYXJlIGRlZmluaXRlbHkgbGFiZWxpbmcgbW9yZSBjZWxscyB0aGFuIHdlIHdlcmUgcHJldmlvdXNseSBhbmQgaW4gc29tZSBjYXNlcyB3ZSBtYXkgYmUgYXNzaWduaW5nIGEgIm5vcm1hbCIgY2VsbCB0eXBlIHRvIHR1bW9yIGNlbGxzLiAKVGhpcyBpcyBub3QgdG90YWxseSBzdXJwcmlzaW5nIGFzIG1hbnkgc29saWQgdHVtb3JzIGhhdmUgY2VsbHMgdGhhdCByZXNlbWJsZSBmaWJyb2JsYXN0cyBhbmQgbXVzY2xlIGNlbGxzLiAKV2UgYWxzbyBzZWUgdmFyaW91cyBwcm9nZW5pdG9ycyBhbmQgSFNDcyBiZWluZyBsYWJlbGVkIGluIHRoZSBsZXVrZW1pYXMgd2hpY2ggcmVmbGVjdCBob3cgbGV1a2VtaWEgY2VsbHMgcmVzZW1ibGUgdGhvc2UgY2VsbCB0eXBlcy4gCk92ZXJhbGwsIEkgdGhpbmsgYWRkaW5nIGluIGBTQ2ltaWxhcml0eWAgaXMgZ2l2aW5nIHVzIGEgbG90IG1vcmUgaW5mb3JtYXRpb24gdGhhbiB3aXRob3V0IGl0LiAKCiMjIFNlc3Npb24gaW5mbyAKCmBgYHtyIHNlc3Npb24gaW5mb30KIyByZWNvcmQgdGhlIHZlcnNpb25zIG9mIHRoZSBwYWNrYWdlcyB1c2VkIGluIHRoaXMgYW5hbHlzaXMgYW5kIG90aGVyIGVudmlyb25tZW50IGluZm9ybWF0aW9uCnNlc3Npb25JbmZvKCkKYGBgCgo=